<<<<<<< HEAD
package com.car.crawler;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.car.biz.AutoHomeBiz;
import com.car.dao.AutoHomeDao;
import com.car.entity.AutoHomeEntity;
import com.car.util.DocUtil;
import com.car.util.Util;

public class AutoHome extends BaseClass{
	
	int estimateCount = 0,priceCount = 0;
	/**
	 * urlType：url类型
	 * 0:主入口
	 * 1：获取所有id
	 * 2：获取口碑所有url
	 * 3：获取bbs所有url
	 * 4：
	 * 5：
	 * 根据类型判断调用哪个方法
	 */
	@Override
	public void start() {
		this.addOneUrl("http://k.autohome.com.cn/2147/index_1.html", 4);
		long start = System.currentTimeMillis();
		logger.info("start---------------------------------------------");
		this.threadRun();

		while (true) {
			if (this.notCrawlurlSet.isEmpty() && Thread.activeCount() == 1
					|| this.waitThreadCount == this.threadCount) {
				long end = System.currentTimeMillis();
				logger.info("总共爬了" + allurlSet.size() + "个网页");
				logger.info("总共耗时" + (end - start) / 1000 + "秒");
				logger.info("end---------------------------------------------");
				System.exit(1);
				// break;
			}

		}
		
	}

	public AutoHome(){
		
	}
	
	public AutoHome(int threadCount){
		this.setThreadCount(threadCount);
	}
	
	@Override
	public void threadRun() {
		for (int i = 0; i < this.getThreadCount(); i++) {
			new Thread(new Runnable() {
				public void run() {
					while (true) {
						String url = getUrl();
						if (url != null) {
							switch(urlType.get(url)){
							case 0:getHome(url);
							break;
							case 1:getAllId(url);
							break;
							case 2:getIwomUrl(url);
							break;
							case 3:getBbsUrl(url);
							break;
							case 4:getIwom(url);
							break;
							}
						} else {
							threadWait();
						}
					}
				}
			}, "thread-" + i).start();
		}
	}
	
	public synchronized void addOneUrl(String url,int type) {
		notCrawlurlSet.add(url);
		allurlSet.add(url);
		urlType.put(url, type);
	}
	
	/**
	 * 1 汽车之家：根据给定的链接获取车系id、品牌名称、车系名称
	 * @param url
	 */
	private void getAllId(String url){
		Document doc = DocUtil.getDocFromTextPage(url, false, false);
		AutoHomeEntity autoHomeEntity = new AutoHomeEntity();
		AutoHomeDao autoHomeDao = new AutoHomeDao();
		Elements es = doc.select("body dl");
		for (Element e : es) {
			String brand = e.select("dt div a").text();
			Elements es2 = e.select("dd ul li");
			for (Element e2 : es2) {
				String id = e2.attr("id");
				String carName = e2.select("h4").text();
				String iwom = null;
				String bbs = null;
				Elements e3 = e2.select("a");
				for (Element e4 : e3) {
					String href = e4.attr("href");
					// logger.info(e4txt);
					if (Util.isMatched(href, "^http://k\\..*")) {
						iwom = Util.getMatched(href, "\\d{2,4}").group();
						//将口碑id加入队列
						addOneUrl(iwom,2);
					} else if (Util.isMatched(href, "^http://club\\..*")) {
						bbs = Util.getMatched(href, "\\d{2,4}").group();
						//将bbsid加入队列
						addOneUrl(bbs,3);
					}
				}
				if (id.length() > 0 || !"".equals(id)) {
					// id = Util.getMatched(id, "\\d{2,4}").group();
					autoHomeEntity.setCarBrand(brand);
					autoHomeEntity.setCarId(id);
					autoHomeEntity.setCarName(carName);
					autoHomeEntity.setIwomId(iwom);
					autoHomeEntity.setBbsId(bbs);
					autoHomeDao.addCarInfo(autoHomeEntity);
					
					logger.info(carName + ",id:" + id + ",brand" + brand
							+ "iwom:" + iwom + "bbs:" + bbs);

				}
			}

		}
		threadNotify();
	}
	/**
	 * 2 根据所给的口碑id 获取该口碑所有页面的url，并将url加入队列
	 * @param iwomId
	 */
	private void getIwomUrl(String iwomId){
		int pages = Util.getAutoHomeKbPage(iwomId);
		for (int i = 1; i <= pages; i++) {
			String url = "http://k.autohome.com.cn/" +iwomId+ "/index_" + i + ".html";
			addOneUrl(url,4);
		}
		threadNotify();
		logger.info("getEstimate");
	}
	/**
	 * 3根据所给的bbsid 获得所有bbs所有页面url，并将url加入队列
	 * @param bbsid
	 */
	private void getBbsUrl(String bbsid){
		
	}
	/**
	 * 4 获取口碑数据
	 * @param url
	 */
	private void getIwom(String url){
		logger.info("getIwom urltype=4");
		Document doc = getDoc(url);
		AutoHomeBiz autoHomeBiz = new AutoHomeBiz();
		if(!autoHomeBiz.addKb(doc)){
			logger.info("addkb wrong thread wait");
			threadWait();
			
		}
	}
	
	
	/**
	 * 从给定的入口获得所有车型入口，并加入队列
	 * @param url
	 */
	private void getHome(String url){
		Elements es = getDoc(url).select(".uibox");
		for (Element e:es){
			String page = e.attr("id");
			if(page.length() == 4){
				addOneUrl("http://www.autohome.com.cn/grade/carhtml/"+page.substring(3)+".html", 1);
				threadNotify();
			}
		}
		
	}
	
	public static void main(String args[]){
		new AutoHome().start();
	}


}
=======
package com.car.crawler;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.car.biz.AutoHomeBiz;
import com.car.dao.AutoHomeDao;
import com.car.entity.AutoHomeEntity;
import com.car.util.DocUtil;
import com.car.util.Util;

public class AutoHome extends BaseClass{
	
	int estimateCount = 0,priceCount = 0;
	/**
	 * urlType：url类型
	 * 0:主入口
	 * 1：获取所有id
	 * 2：获取口碑所有url
	 * 3：获取bbs所有url
	 * 4：
	 * 5：
	 * 根据类型判断调用哪个方法
	 */
	@Override
	public void start() {
		this.addOneUrl("http://k.autohome.com.cn/2147/index_1.html", 4);
		long start = System.currentTimeMillis();
		logger.info("start---------------------------------------------");
		this.threadRun();

		while (true) {
			if (this.notCrawlurlSet.isEmpty() && Thread.activeCount() == 1
					|| this.waitThreadCount == this.threadCount) {
				long end = System.currentTimeMillis();
				logger.info("总共爬了" + allurlSet.size() + "个网页");
				logger.info("总共耗时" + (end - start) / 1000 + "秒");
				logger.info("end---------------------------------------------");
				System.exit(1);
				// break;
			}

		}
		
	}

	public AutoHome(){
		
	}
	
	public AutoHome(int threadCount){
		this.setThreadCount(threadCount);
	}
	
	@Override
	public void threadRun() {
		for (int i = 0; i < this.getThreadCount(); i++) {
			new Thread(new Runnable() {
				public void run() {
					while (true) {
						String url = getUrl();
						if (url != null) {
							switch(urlType.get(url)){
							case 0:getHome(url);
							break;
							case 1:getAllId(url);
							break;
							case 2:getIwomUrl(url);
							break;
							case 3:getBbsUrl(url);
							break;
							case 4:getIwom(url);
							break;
							}
						} else {
							threadWait();
						}
					}
				}
			}, "thread-" + i).start();
		}
	}
	
	public synchronized void addOneUrl(String url,int type) {
		notCrawlurlSet.add(url);
		allurlSet.add(url);
		urlType.put(url, type);
	}
	
	/**
	 * 1 汽车之家：根据给定的链接获取车系id、品牌名称、车系名称
	 * @param url
	 */
	private void getAllId(String url){
		Document doc = DocUtil.getDocFromTextPage(url, false, false);
		AutoHomeEntity autoHomeEntity = new AutoHomeEntity();
		AutoHomeDao autoHomeDao = new AutoHomeDao();
		Elements es = doc.select("body dl");
		for (Element e : es) {
			String brand = e.select("dt div a").text();
			Elements es2 = e.select("dd ul li");
			for (Element e2 : es2) {
				String id = e2.attr("id");
				String carName = e2.select("h4").text();
				String iwom = null;
				String bbs = null;
				Elements e3 = e2.select("a");
				for (Element e4 : e3) {
					String href = e4.attr("href");
					// logger.info(e4txt);
					if (Util.isMatched(href, "^http://k\\..*")) {
						iwom = Util.getMatched(href, "\\d{2,4}").group();
						//将口碑id加入队列
						addOneUrl(iwom,2);
					} else if (Util.isMatched(href, "^http://club\\..*")) {
						bbs = Util.getMatched(href, "\\d{2,4}").group();
						//将bbsid加入队列
						addOneUrl(bbs,3);
					}
				}
				if (id.length() > 0 || !"".equals(id)) {
					// id = Util.getMatched(id, "\\d{2,4}").group();
					autoHomeEntity.setCarBrand(brand);
					autoHomeEntity.setCarId(id);
					autoHomeEntity.setCarName(carName);
					autoHomeEntity.setIwomId(iwom);
					autoHomeEntity.setBbsId(bbs);
					autoHomeDao.addCarInfo(autoHomeEntity);
					
					logger.info(carName + ",id:" + id + ",brand" + brand
							+ "iwom:" + iwom + "bbs:" + bbs);

				}
			}

		}
		threadNotify();
	}
	/**
	 * 2 根据所给的口碑id 获取该口碑所有页面的url，并将url加入队列
	 * @param iwomId
	 */
	private void getIwomUrl(String iwomId){
		int pages = Util.getAutoHomeKbPage(iwomId);
		for (int i = 1; i <= pages; i++) {
			String url = "http://k.autohome.com.cn/" +iwomId+ "/index_" + i + ".html";
			addOneUrl(url,4);
		}
		threadNotify();
		logger.info("getEstimate");
	}
	/**
	 * 3根据所给的bbsid 获得所有bbs所有页面url，并将url加入队列
	 * @param bbsid
	 */
	private void getBbsUrl(String bbsid){
		
	}
	/**
	 * 4 获取口碑数据
	 * @param url
	 */
	private void getIwom(String url){
		logger.info("getIwom urltype=4");
		Document doc = getDoc(url);
		AutoHomeBiz autoHomeBiz = new AutoHomeBiz();
		if(!autoHomeBiz.addKb(doc)){
			logger.info("addkb wrong thread wait");
			threadWait();
			
		}
	}
	
	
	/**
	 * 从给定的入口获得所有车型入口，并加入队列
	 * @param url
	 */
	private void getHome(String url){
		Elements es = getDoc(url).select(".uibox");
		for (Element e:es){
			String page = e.attr("id");
			if(page.length() == 4){
				addOneUrl("http://www.autohome.com.cn/grade/carhtml/"+page.substring(3)+".html", 1);
				threadNotify();
			}
		}
		
	}
	
	public static void main(String args[]){
		new AutoHome().start();
	}


}
>>>>>>> 12cebf820d4efd66d730623df0b55f6b98421d88
